package com.doit.shark.crawler.jditem.phone;/**
 * Created by hunter.coder 涛哥
 * 2019/4/15 16:16
 * 交流qq:657270652
 * Version: 1.0
 * 更多学习资料：https://blog.csdn.net/coderblack/
 * Description:
 **/

import com.hankcs.hanlp.HanLP;
import org.apache.commons.lang3.StringUtils;
import org.codehaus.jackson.map.ObjectMapper;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;

/**
 * @Author hunter.coder
 * @Date 2019/4/15 16:16
 * @Version 1.0
 * @Description TODO
 **/
public class UrlExtractor {

    public static void main(String[] args) throws Exception {

        String cat1 = "手机通讯/数码";
        String cat2 = "手机通讯";
        String cat3 = "手机";

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        String dtStr = sdf.format(new Date());
        BufferedWriter bw = new BufferedWriter(new FileWriter("G:\\data_shark\\testdata\\jd_phone_content\\" + dtStr + ".dat"));

        ObjectMapper om = new ObjectMapper();

        for(int i=1;i<=128;i++) {
            System.out.println("-----------------"+i+"---------------");
            Document doc = Jsoup.connect("https://list.jd.com/list.html?cat=9987,653,655&page=" + i).get();
            Element ul = doc.getElementsByClass("gl-warp clearfix").get(0);

            Elements aEles = ul.select("a[title='']");
            for (Element aEle : aEles) {

                String href = aEle.attr("href");
                String title = aEle.child(0).text();

                List<String> kwds = HanLP.extractKeyword(title, 5);

                HashMap<String, String> infoMap = new HashMap<>();
                infoMap.put("cat1",cat1);
                infoMap.put("cat2",cat2);
                infoMap.put("cat3",cat3);
                infoMap.put("title",title);
                infoMap.put("url",href);
                infoMap.put("kwds", StringUtils.join(kwds," "));

                bw.write(om.writeValueAsString(infoMap));
                bw.newLine();
                bw.flush();

            }
        }

        bw.close();

    }
}
